from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.selector import Selector
import time

class ExtTest(object):
    def __init__(self,stats):
            self.stats = stats
            self.stats.set_value('pages_crawled',0)
            
            self.stats.set_value('store_links',0)
    @classmethod      
    def from_crawler(cls,crawler):
        if not crawler.settings.getbool('MYEXT_ENABLED'):
            raise NotConfigured
        ext = cls(crawler.stats)
        crawler.signals.connect(ext.spider_opened,signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        crawler.signals.connect(ext.response_received,signal=signals.response_received)
        crawler.signals.connect(ext.item_scraped,signal=signals.item_scraped)
        return ext
    def spider_opened(self,spider):
#         print "***********extension Spider open ********************"
        self.stats.set_value('start',time.time())
        
        
    def spider_closed(self,spider):
        spendtime = time.time() - self.stats.get_value('start')
#         print "**********extension Spider closed*************"
#         print "pages crawled:",self.stats.get_value('pages_crawled')
#         print "time spent :%.5f s" % spendtime
#         print "stored links: %d" % self.stats.get_value('store_links')
        
    def response_received(self,response,request,spider):
        self.stats.inc_value('pages_crawled')
#         if self.stats.get_value('pages_crawled') % 20 == 0:
#             print "Spider has crawled %d pages ......" % self.stats.get_value('pages_crawled')            
    
    def item_scraped(self,item,response,spider):
        self.stats.inc_value('store_links')
#         if response.url=="http://scan.com/xss.php":
#             sel=Selector(response)
#             forms=sel.xpath("//form")
#             for form in forms:
#                 input_vector=form.xpath("input[@type='text']")[0]
#                 submit=form.xpath("input[@type='submit']")[0]
#                 print input_vector
#                 print submit